#!/usr/bin/perl
use strict;
use warnings;
use CGI;
use DBI;
use Getopt::Long;
use HTML::TokeParser;
use JSON;
use POSIX;
use Switch;
use Time::HiRes qw( gettimeofday tv_interval );
use WWW::Mechanize;

# Timestamps
my $file_ts   = time();
my $starttime = [gettimeofday];
my $endtime;

# Stats
my $duration;
my $perf = 0;

# Crawl count
my $records = 0;
my $crawled = 0;

# Base url
my $url = 'http://hypem.com/';

# Default option values
my $ctype = 'home';    # Crawl type: home, search, or both
my $limit = 100;          # Page limit
my $delay = 0;           # Adjust crawl rate by setting sleep time in ms

# Log script startup
print "Start time: " . calc_ts() . "\n\n";
print
"CONFIG:\n  Crawl type: $ctype\n  Page limit: $limit\n  Sleep time: $delay\n\n";

# Set mysql-related properties here
my $dbname   = 'hypem';
my $username = 'user';
my $password = 'password';

GetOptions(
    'type=s'  => \$ctype,
    'limit=i' => \$limit,
    'delay=i' => \$delay
) or exit;

arg_check();

my $mech = WWW::Mechanize->new(
    agent   => 'Mozilla/5.0 (compatible; kurtbot 1.0; http://ens8-ble.com)',
    onerror => undef
);    # Initialize robot

my $dbh = DBI->connect( "DBI:mysql:$dbname", $username, $password )
  || die
  "Could not connect to database: $DBI::errstr";   # Open connection to database

if ( $ctype =~ m/^(home|both)$/i ) {
    print "Begin scrape from the homepage...\n";
    navigate( '', 'home' );
}

if ( $ctype =~ m/^(search|both)$/i ) {
    my (@queries) = (
'SELECT artist FROM (SELECT artist, COUNT(1) posts FROM mp3s WHERE artist != " " GROUP BY artist) AS popular WHERE POSTS > 20 ORDER BY posts DESC',
'SELECT artist FROM (SELECT artist, SUM(favcount) fsum FROM mp3s GROUP BY artist) AS favcounts WHERE fsum > 500 ORDER BY fsum DESC'
    );

    foreach (@queries) {
        print "Using query $_\n";
        my $sth = $dbh->prepare($_)
          or die "Couldn't prepare statement: " . $dbh->errstr();
        $sth->execute()
          or die "Couldn't execute statement: " . $sth->errstr();

        my $popost;
        $sth->bind_columns( \$popost );

        while ( $sth->fetch ) {
            my $popath = "search/$popost";

            # Regex substitution for special characters, only "+" so far -> TDB: translate spec. characters to ASCII hex code
            $popath =~ s/\+//g;
            #

            print "Begin scrape for \"$popost\"...\n";
            navigate( $popath, 'search' );
        }
    }
}

$endtime  = [gettimeofday];                     # Set endtime
$duration = tv_interval $starttime, $endtime;
$perf     = $duration / $crawled if ($crawled != 0);
print "Records inserted/updated: $records\nPages crawled: $crawled\nEndtime: "
  . calc_ts()
  . "\n\n";                                     # Notify completion of run
print "Duration: $duration secs\n";
print "Performance: $perf secs per page\n";
$dbh->disconnect();                             # Disconnect from database

sub navigate {
    my $count = 1;
    my $spath = $_[0];
    my $stype = $_[1];
    my $page  = "$url" . "$spath";

    $mech->get($page);

    while ( $mech->{content} =~ /alt="Next Page"/ && $count < $limit ) {
        if ( $mech->status() != '200' ) {
            print "Script exits at page "
              . $count
              . "with a "
              . $mech->status()
              . " status code.";
            last;
        }
        print " $page: " . $count++ . "\n";
        scrape( $mech->{content} );
        $crawled++;    # Increment total pages crawled
        switch ($stype) {
            $mech->follow_link( url_regex => qr/\/$count\// ) case 'home';
            $mech->follow_link( url_regex => qr/\/$spath\/$count\// ) case
              'search';
        }
        sleep($delay);
    }
    print " $page: " . $count++ . "\n\n";
    scrape( $mech->{content} );
    $crawled++;
}

sub scrape {
    my $page_content = $_[0];
    my $tp = HTML::TokeParser->new( \$page_content ) || die "Can't open: $!";

    my (
        $artist,      $song,     $id,       $postid,
        $song_length, $favcount, $is_alive, $timestamp,
        $amazon,      $emusic,   $itunes,   $link
    );

    while ( my $token = $tp->get_token ) {
        my $ttype = shift @{$token};

        if ( $ttype eq "T" ) {
            my ($text) = @{$token};

            if ( $text =~ m/trackList\[document.location.href\].push/ ) {
                $text =~ s/trackList\[document.location.href\]\.push\(|\)\;//g;
                $text =~ s/\s+/ /g;

                my $json = new JSON;
                my $json_text =
                  $json->allow_nonref->utf8->relaxed->escape_slash->loose
                  ->allow_singlequote->allow_barekey->decode($text);

                $artist      = $json_text->{artist};
                $song        = $json_text->{song};
                $id          = $json_text->{id};
                $postid      = $json_text->{postid};
                $song_length = $json_text->{time};
                $timestamp   = $json_text->{ts};
                $amazon      = $json_text->{amazon};
                $emusic      = $json_text->{emusic};
                $itunes      = $json_text->{itunes};

                $artist =~ s/\'/\\'/g;
                $song   =~ s/\'/\\'/g;

            }
        }
        elsif ( $ttype eq "S" )    # start tag?
        {
            my ( $tag, $attr, $attrseq, $rawtxt ) = @{$token};

            if ( $rawtxt =~ m/class="favcount-off/ ) {
                $favcount = $tp->get_trimmed_text('/a');
                $is_alive = 0;
            }
            elsif ( $rawtxt =~ m/class="play-ctrl play"/ ) {
                $is_alive = 1;
            }
            elsif ( $rawtxt =~ m/Read this post:/ ) {
                $link = $token->[1]{href} || "-";
                
#				print "$id $artist - $song\n Inserting into mp3s...\n";
				$dbh->do(
"INSERT INTO mp3s (id, postid, artist, song, song_length, favcount, date_added) VALUES('$id', '$postid', '$artist', '$song', (select sec_to_time('$song_length')), $favcount, CURRENT_TIMESTAMP) ON DUPLICATE KEY UPDATE favcount=$favcount, last_update=CURRENT_TIMESTAMP"
				);

#				print " Inserting into stores...\n";
				$dbh->do(
"INSERT INTO stores (id, amazon, emusic, itunes, date_added) VALUES('$id', '$amazon', '$emusic', '$itunes', CURRENT_TIMESTAMP) ON DUPLICATE KEY UPDATE amazon='$amazon', emusic='$emusic', itunes='$itunes', last_update=CURRENT_TIMESTAMP"
				);
				
#				print(" Inserting into blog_info...\n");
				$dbh->do(
"INSERT INTO blog_info (id, is_alive, blog_ts, link, date_added) VALUES('$id', $is_alive, (select from_unixtime('$timestamp')), '$link', CURRENT_TIMESTAMP) ON DUPLICATE KEY UPDATE is_alive=$is_alive, last_update=CURRENT_TIMESTAMP"
				);
				
                $records++;
            }
        }
        elsif ( $ttype eq "C" )    # comment?
        {
        }
        elsif ( $ttype eq "E" )    # end tag?
        {
        }
        elsif ( $ttype eq "D" )    # declaration?
        {
        }
    }
}

sub arg_check {
    my $fail = 0;

    if ( $ctype !~ m/^(home|search|both)$/i ) {
        print "Type must be 'home', 'search', or 'both'.\n";
        $fail = 1;
    }
    if ( $limit <= 0 || $limit > 300 ) {
        print "Limit must be an integer between 1 and 300, inclusive.\n";
        $fail = 1;
    }
    if ( $delay < 0 ) {
        print "Delay must be a positive integer.\n";
        $fail = 1;
    }

    if ( $fail == 1 ) {
        print "See README.txt for more details.\n";
        exit;
    }

}

sub calc_ts {
    my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) =
      localtime();
    return POSIX::strftime( "%Y-%m-%d %H:%M:%S",
        $sec, $min, $hour, $mday, $mon, $year );
}